<?php 
require_once "../simple_html_dom/simple_html_dom.php";//引入dom树分析包
require_once "../php-readability-master/lib/Readability.inc.php";//引入正文提取分析包
class cwantSpider{
	private $resultUrl;
	public function __construct(){
		$this->resultUrl=array();
	}
	public function getExt($url){//获取url域名
		$arr = parse_url($url);     
		
		return $arr['scheme']."://".$arr['host'];
	}
	public function crawlKeywordList($keywordArray,$url){//爬某一个链接内跟所给数组内关键词有关的链接
			if(!empty($url))
			{
				//$this->resultUrl[]=$url;
				$html = file_get_html($url);
				if($html!=""){
					foreach($html->find('a') as $e){
							$link=$e->href;
							if(!strstr($link,"http://")&&!strstr($link,"https://")){
								$link=$this->getExt($url).$link;
							}
							$content=$e->plaintext;
							if($this->is_relative($content,$keywordArray)){//判断链接是否含有所需关键词
								if(!in_array($link,$this->resultUrl)){
									$this->resultUrl[]=$link;
								}
								
							}
						
						}
				}
				if(is_object($html)){
					$html->clear(); 
					unset($html);//清空缓存
				}
				
			}
			
	}
	public function getRelativeUrl($keywordArray,$urlList){
		foreach ($urlList as $key => $value) {
			$this->crawlKeywordList($keywordArray,$value);
		}
		return $this->resultUrl;

	}
	
	public function is_relative($content,$keywordArray){//判断关键词与该链接是否有关系
		foreach ($keywordArray as $key => $value) {
			if($content!=""&&$value!="")
			{
				if(strstr($content,$value)){
				return 1;
				}
			}
			
		}

		return 0;
	}
	public function getAllContent($url_array){//并发处理所有的，目标url并获得所有的值
		$handles = $contents = array(); 
		//初始化curl multi对象
		$mh = curl_multi_init();
		//添加curl 批处理会话
		foreach($url_array as $key => $url)
		{
		    $handles[$key] = curl_init($url);
		    curl_setopt($handles[$key], CURLOPT_RETURNTRANSFER, 1);
		    curl_setopt($handles[$key], CURLOPT_TIMEOUT, 10);
		    curl_multi_add_handle($mh, $handles[$key]);
		}
		//======================执行批处理句柄=================================
		$active = null;
		do {
		    $mrc = curl_multi_exec($mh, $active);
		} while ($mrc == CURLM_CALL_MULTI_PERFORM);

		while ($active and $mrc == CURLM_OK) {
		    if(curl_multi_select($mh) === -1){
		       usleep(100);
		    }
		    do {
		        $mrc = curl_multi_exec($mh, $active);
		    } while ($mrc == CURLM_CALL_MULTI_PERFORM);
		}
		//====================================================================
		//获取批处理内容
		foreach($handles as $i => $ch)//$handles为所有执行的任务，里面存储的是各网址
		{
		    $content = curl_multi_getcontent($ch);
		    $contents[$i]['content'] = curl_errno($ch) == 0 ? $content : '';
		    $contents[$i]['url']=$url_array[$i];
		}
		//移除批处理句柄
		foreach($handles as $ch)
		{
		    curl_multi_remove_handle($mh, $ch);
		}
		//关闭批处理句柄
		curl_multi_close($mh);
		return $contents;
	}
	public function getContent($urlList){//爬所有链接的内容并整理
			$result=array();
			$contents=$this->getAllContent($urlList);
			$i=0;
			foreach ($contents as $key => $html) {
				if(!empty($html['content'])){
					$Readability= new Readability($html['content']); // default charset is utf-8
					$ReadabilityData = $Readability->getContent();
					$result[$i]['content']=$ReadabilityData;
					$result[$i]['url']=$html['url'];
					$i++;
				}
			}
		return $result;
		
	}
	public function clear(){
		unset($this->keywordArray);
		unset($this->resultUrl);
		unset($this->initUrlDomain);
	}
	//public function getNews($key)
}
?>
